﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.Web;
using System.Data;
using System.Text.RegularExpressions;
using System.IO;
using System.Windows.Forms;
using System.Collections;

using HtmlAgilityPack;

namespace WBrowser
{
    //internal class AcceptAllCertificatePolicy : System.Net.ICertificatePolicy
    //{
    //    public AcceptAllCertificatePolicy() { }

    //    public bool CheckValidationResult(System.Net.ServicePoint sPoint, System.Security.Cryptography.X509Certificates.X509Certificate cert, System.Net.WebRequest wRequest, int certProb)
    //    {
    //        // Always accept
    //        return true;
    //    }
    //}

    public class BrowserSession
    {
        public bool _isPost;
        public HtmlWeb _htmlWeb = new HtmlWeb();
        public HttpWebRequest _request;
        public HttpWebResponse _response;
        private bool _isLogged;

        public BrowserSession()
        {
            MITI.BaseDBCe.g_ConnectStr = GeneralClass.ConnectString();
            FormElements = new FormElementCollection();
        }

        #region Profile Properties
        
        public DataRow pTemplateRow { get; set; }
        public DataRow pPostsRow { get; set; }
        public DataRow pForumPostsRow { get; set; }
        public DataRow pThreadTemplateRow { get; set; }
        //public DataTable pThreadTemplateTable { get; set; }
        
        /// <summary>
        /// System.Net.CookieCollection. Provides a collection container for instances of Cookie class 
        /// </summary>
        public HtmlAgilityPack.HtmlWeb Web
        { 
            get { return _htmlWeb; }
            set { _htmlWeb = value; } 
        }

        /// <summary>
        /// System.Net.CookieCollection. Provides a collection container for instances of Cookie class 
        /// </summary>
        public CookieCollection Cookies { get; set; }

        /// <summary>
        /// Provide a key-value-pair collection of form elements 
        /// </summary>
        public FormElementCollection FormElements { get; set; }

        /// <summary>
        /// Templates Object
        /// </summary>
        public Templates ObjTemplates { get; set; }

        #endregion

        #region base functions
        /// <summary>
        /// Makes a HTTP GET request to the given URL
        /// </summary>
        public HtmlAgilityPack.HtmlDocument Get(string url)
        {
            _isPost = false;
            _htmlWeb = CreateWebRequestObject();
            HtmlAgilityPack.HtmlDocument doc = _htmlWeb.Load(url);
            return doc;
        }

        /// <summary>
        /// Makes a HTTP POST request to the given URL
        /// </summary>
        public HtmlAgilityPack.HtmlDocument Post(string url)
        {
            _isPost = true;
            _htmlWeb = CreateWebRequestObject();
            HtmlAgilityPack.HtmlDocument doc = _htmlWeb.Load(url, "POST");
            return doc;
        }

        /// <summary>
        /// Creates the HtmlWeb object and initializes all event handlers. 
        /// </summary>
        public HtmlWeb CreateWebRequestObject()
        {
            HtmlWeb web = new HtmlWeb();
            web.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13 GTBDFff GTB7.0";
            web.Accept = " text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8, */*";
            web.UseKeepAlive = false;
            web.ContentType = @"application/x-www-form-urlencoded";
            web.UseCookies = true;
            web.ForcedTimeout = 100;
            web.PreRequest = new HtmlWeb.PreRequestHandler(OnPreRequest);
            web.PostResponse = new HtmlWeb.PostResponseHandler(OnAfterResponse);
            web.PreHandleDocument = new HtmlWeb.PreHandleDocumentHandler(OnPreHandleDocument);
            return web;
        }

        /// <summary>
        /// Event handler for HtmlWeb.PreRequestHandler. Occurs before an HTTP request is executed.
        /// </summary>
        public bool OnPreRequest(HttpWebRequest request)
        {
            _request = request;
            AddCookiesTo();               // Add cookies that were saved from previous requests
            if (_isPost)
                AddPostDataTo();         // We only need to add post data on a POST request
            return true;
        }

        /// <summary>
        /// Event handler for HtmlWeb.PostResponseHandler. Occurs after a HTTP response is received
        /// </summary>
        public void OnAfterResponse(HttpWebRequest request, HttpWebResponse response)
        {
            _request = request;
            _response = response;
            //update last cookies sent by server
            //string[] keys = response.Headers.GetValues("Set-Cookie");
            //GetCookies(keys);
            if (_isPost) //If request is POST then saving
                SaveCookiesFrom(); // Save cookies for subsequent requests
        }

        /// <summary>
        /// Event handler for HtmlWeb.PreHandleDocumentHandler. Occurs before a HTML document is handled
        /// </summary>
        public void OnPreHandleDocument(HtmlAgilityPack.HtmlDocument document)
        {
            //SaveHtmlDocument(document);
            //Close respone
            _response.Close();
        }

        /// <summary>
        /// Assembles the Post data and attaches to the request object
        /// </summary>
        public void AddPostDataTo()
        {
            string postData = "";
            try
            {
                postData = FormElements.AssemblePostPayload();
                byte[] buff = Encoding.UTF8.GetBytes(postData.ToCharArray());
                _request.ContentLength = buff.Length;

                //_request.UserAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13 GTBDFff GTB7.0";
                //_request.Accept = " text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8, */*";
                //_request.KeepAlive = false;
                //_request.ContentType = @"application/x-www-form-urlencoded";

                Stream reqStream = _request.GetRequestStream();
                reqStream.Write(buff, 0, buff.Length);
                reqStream.Close();
            }
            catch (Exception ex)
            {
                //Console.WriteLine(ex.Message);
                throw ex;
            }

            //try
            //{
            //    string postdata = string.Format(postData);
            //    byte[] postBuffer = Encoding.GetEncoding(1252).GetBytes(postData);
            //    request.ContentLength = postBuffer.Length;
            //    Stream postDataStream = request.GetRequestStream();
            //    postDataStream.Write(postBuffer, 0, postBuffer.Length);
            //    postDataStream.Close();
            //}
            //catch (Exception ex)
            //{
            //    Console.WriteLine(ex.Message);

            //}
        }

        /// <summary>
        /// Add cookies to the request object
        /// </summary>
        public void AddCookiesTo()
        {
            if (Cookies != null && Cookies.Count > 0)
            {
                _request.CookieContainer.Add(Cookies);
            }
        }

        /// <summary>
        /// Saves cookies from the response object to the local CookieCollection object
        /// </summary>
        public void SaveCookiesFrom()
        {
            //check if response object has any cookies or not
            if (_response.Cookies.Count > 0)
            {
                //check if this is the first request/response, if this is the response of first request gCookies
                //will be null
                if (Cookies == null)
                {
                    Cookies = _response.Cookies;
                }
                else
                {
                    foreach (Cookie oRespCookie in _response.Cookies)
                    {
                        bool bMatch = false;
                        foreach (Cookie oReqCookie in this.Cookies)
                        {
                            if (oReqCookie.Name == oRespCookie.Name)
                            {
                                oReqCookie.Value = oRespCookie.Value;
                                bMatch = true;
                                break; // 
                            }
                        }
                        if (!bMatch)
                            this.Cookies.Add(oRespCookie);
                    }
                }
            }
            else
                Cookies = null;
        }


        /// <summary>
        /// Saves the form elements collection by parsing the HTML document
        /// </summary>
        public void SaveHtmlDocument(HtmlAgilityPack.HtmlDocument document)
        {
            //_htmlDoc = document;
            //FormElements = new FormElementCollection(_htmlDoc);
        }

#endregion

        #region "Functions"

        /// <summary>
        /// Init params of POST  to login a forum.
        /// </summary>
        public string InitLoginParams()
        {
            string url = "";
            if (pTemplateRow["type"].ToString() == "vBulletin")
            {
                //Method 1:
                this.FormElements["vb_login_username"] = pTemplateRow["username"].ToString();
                this.FormElements["cookieuser"] = "checked";
                this.FormElements["vb_login_password"] = pTemplateRow["password"].ToString();
                this.FormElements["securitytoken"] = "guest";
                this.FormElements["do"] = "login";
                url = pTemplateRow["url"] + @"/login.php?do=login";
            }
            else if (pTemplateRow["type"].ToString() == "IPB")
            {
                //Method 1
                this.FormElements["referer"] = "";
                this.FormElements["username"] = pTemplateRow["username"].ToString();
                this.FormElements["password"] = pTemplateRow["password"].ToString();
                this.FormElements["rememberMe"] = "1";
                url = pTemplateRow["url"] + @"/index.php?app=core&module=global&section=login&do=process";
            }
            else if (pTemplateRow["type"].ToString() == "phpbb3")
            {
                //Method 1
                this.FormElements["username"] = pTemplateRow["username"].ToString();
                this.FormElements["password"] = pTemplateRow["password"].ToString();
                this.FormElements["autologin"] = "on";
                this.FormElements["sid"] = "";
                this.FormElements["redirect"] = "";// "index.php";
                this.FormElements["login"] = "Login";
                url = pTemplateRow["url"] + @"/ucp.php?mode=login";
            }
            else if (pTemplateRow["type"].ToString() == "phpbb2")
            {
                //Method 1
                this.FormElements["username"] = pTemplateRow["username"].ToString();
                this.FormElements["password"] = pTemplateRow["password"].ToString();
                this.FormElements["autologin"] = "on";
                this.FormElements["redirect"] = "";
                this.FormElements["login"] = "Log in";
                url = pTemplateRow["url"] + @"/login.php";
            }
            return url;
        }

        /// <summary>
        /// Login in a forum. If susscessful set _isLogged = true, else false
        /// </summary>
        public void DoLogin()
        {
            _isLogged = false;
            try
            {
                //Init login params
                string url = InitLoginParams();

                HtmlAgilityPack.HtmlDocument doc = Post(url);
                pTemplateRow["html"] = doc.DocumentNode.InnerHtml;
                //REMOVE SOME TAGS
                //html = ScrubHTML(doc);

                int count = 0;
                string value = "";
                List<string> key = new List<string>();
                string pattern = @"((T|t)hank (Y|y)ou|(S|s)uccessful(ly|) logged in|(L|l)og (O|o)ut|" + pTemplateRow["username"] + ")";
                Regex r = new Regex(pattern);
                Match m = r.Match(pTemplateRow["html"].ToString());
                while (m.Success)
                {
                    value = m.Value.ToLower().Trim();
                    if (!key.Contains(value))
                    {
                        key.Add(value);
                        count++;
                    }
                    m = m.NextMatch();
                }

                //If count > 1 successfully log in
                if (count >= 1)
                {
                    pTemplateRow["loginstatus"] = "Correct-" + DateTime.Now.ToString();
                    _isLogged = true;
                }
                else
                {
                    count = 0;
                    value = "";
                    key = new List<string>();
                    pattern = @"((S|s)ign (I|i)n|(L|l)og (I|i)n|(R|r)egister\s*((N|n)ow|)|(F|f)orgot(ten|)\s+(your|my|)\s+(P|p)assword|(Please|)\s+try again|(incorrect|invalid) username|(R|r)emember me)";
                    r = new Regex(pattern);
                    m = r.Match(pTemplateRow["html"].ToString());
                    while (m.Success)
                    {
                        value = m.Value.ToLower().Trim();
                        if (!key.Contains(value))
                        {
                            key.Add(value);
                            count++;
                        }
                        m = m.NextMatch();
                    }

                    //If count > 1 log in invalid username or password
                    if (count > 1)
                    {
                        pTemplateRow["loginstatus"] = "Invalid-" + DateTime.Now.ToString() + "-" + Web.StatusCode;

                    }
                    else //Error log in 
                    {
                        pTemplateRow["loginstatus"] = "Error-" + DateTime.Now.ToString() + "-" + Web.StatusCode;
                    }
                }
            }
            catch (WebException ex)
            {
                pTemplateRow["loginstatus"] = "Fail-" + DateTime.Now.ToString() + "-" + Web.StatusCode + ex.Message;
            }
            catch (Exception e)
            {
                pTemplateRow["loginstatus"] = "Fail-" + DateTime.Now.ToString() + "-" + Web.StatusCode + e.Message;
            }
        }

        /// <summary>
        /// Find url contains your posts list
        /// </summary>
        public void FindPost()
        {
            try
            {
                //Login
                if (!_isLogged)
                    DoLogin();

                if (_isLogged)
                {
                    if (pTemplateRow["myposturl"].ToString() != "")
                    {
                        List<string> list = new List<string>();
                        string url = pTemplateRow["url"].ToString();
                        list = pTemplateRow["myposturl"].ToString().Split(new char[] { '/' }).ToList();
                        foreach (string s in list)
                        {
                            if (!url.Contains(s))
                                url += "/" + s;
                        }

                        HtmlAgilityPack.HtmlDocument doc = Get(url);
                        string htmlcontent = doc.DocumentNode.InnerHtml;
                        if (htmlcontent.ToLower().Contains(pPostsRow["title"].ToString().ToLower()))
                        {
                            pForumPostsRow["poststatus"] = "Found-" + DateTime.Now.ToString();
                        }
                        else
                            pForumPostsRow["poststatus"] = "Not Found-" + DateTime.Now.ToString();
                    }
                    else
                        pForumPostsRow["poststatus"] = "empty myposturl-" + DateTime.Now.ToString();
                }
                else
                    pForumPostsRow["poststatus"] = "Fail Login-" + DateTime.Now.ToString() + "-" + Web.StatusCode;

            }
            catch (WebException ex)
            {
                pForumPostsRow["poststatus"] = "Error-" + DateTime.Now.ToString() + "-" + Web.StatusCode + ex.Message;
            }
        }

        /// <summary>
        /// Find url contains your posts list, include 3 step
        /// Step 1. Load User control panel url
        /// Step 2. Search 'Profile' url
        /// Step 3. If found 'Profile' url, load it
        /// Step 4. Search 'Find all threads ...' url and save it if found
        /// </summary>
        public void FindStatisticsUrl()
        {
            bool isFound = false; 
            string url, xpath;
            HtmlAgilityPack.HtmlDocument doc;
            HtmlNode link;
            try
            {
                //Login
                if (!_isLogged)
                    DoLogin();

                if (pTemplateRow["type"].ToString() == "vBulletin")
                {
                    //Step 1. Load User control panel url
                    url = pTemplateRow["url"].ToString() + "/usercp.php";
                    doc = Get(url);
                    if (doc != null)
                    {
                        //Save html
                        pTemplateRow["html"] = doc.DocumentNode.InnerHtml;
                        //Step 2. Search 'Profile' url
                        //xpath = ".//a[contains(text(),'" + pTemplateRow["username"].ToString() + "') or contains(text(),'Your Profile') or contains(text(),'My Profile')]";
                        xpath = ".//a[";
                        xpath += "contains(text(),'" + pTemplateRow["username"].ToString() + "')";
                        xpath += " or ";
                        xpath += "contains(text(),'Your Profile')";
                        xpath += " or ";
                        xpath += "contains(text(),'My Profile')";
                        xpath += "]";
                        link = doc.DocumentNode.SelectSingleNode(xpath);
                        if (link != null)
                        {
                            //Step 3. If found 'Profile' url, load it
                            url = link.Attributes["href"].Value;
                            if (!url.ToLower().Contains(pTemplateRow["forum_name"].ToString()))
                                url = pTemplateRow["url"] + "/" + url;
                            doc = Get(url);
                            if (doc != null)
                            {
                                //Step 4. Search 'Find all threads ...' url and save it if found
                                pTemplateRow["html"] = doc.DocumentNode.InnerHtml;
                                //link = doc.DocumentNode.SelectSingleNode(".//*[@id='collapseobj_stats']/div/fieldset[1]/ul/li[4]/a");
                                xpath = ".//a[";
                                xpath += "contains(text(),'Find all threads started')";
                                xpath += " or ";
                                xpath += "contains(text(),'Find all started threads')";
                                xpath += " or ";
                                xpath += "contains(text(),'Tìm tất cả các đề tài liên quan')";
                                xpath += "]";
                                link = doc.DocumentNode.SelectSingleNode(xpath);
                                if (link != null)
                                {
                                    url = link.Attributes["href"].Value;
                                    if (url.Contains(pTemplateRow["url"] + "/"))
                                        url = url.Replace(pTemplateRow["url"] + "/", "");

                                    string userid = "";
                                    Regex r = new Regex(@"\d{2,}");
                                    Match m = r.Match(url);
                                    if (m.Success)
                                        userid = m.Value;

                                    pTemplateRow["myposturl"] = url.Replace("&amp;", "&");
                                    pTemplateRow["userid"] = userid;
                                    isFound = true;
                                }
                            }
                        }
                    }
                }
                else if (pTemplateRow["type"].ToString() == "IPB")
                {
                    doc = Get(pTemplateRow["url"].ToString());
                    pTemplateRow["html"] = doc.DocumentNode.InnerHtml;
                    link = doc.DocumentNode.SelectSingleNode(".//*[@id='user_link']");

                    if (link != null)
                    {
                        url = link.Attributes["href"].Value.Replace("&amp;", "&");
                        Regex r = new Regex(@"(index\.php\?showuser=(?<userid>\d+)|user/(?<userid>\d+))");

                        Match m = r.Match(url);
                        //If found userid then continue ...
                        if (m.Success)
                        {
                            //Save userid
                            pTemplateRow["userid"] = m.Result("${userid}");
                            //Open url to find topics
                            doc = Get(url);
                            link = doc.DocumentNode.SelectSingleNode(@".//*[@id='tab_link_forums:topics' or @id='tab_link_topics']//a");
                            if (link != null)
                            {

                                url = link.Attributes["href"].Value.Replace("&amp;", "&");
                                if (url.Contains(pTemplateRow["url"] + "/"))
                                    url = url.Replace(pTemplateRow["url"] + "/", "");
                                pTemplateRow["myposturl"] = url;
                                isFound = true;
                            }
                        }
                    }
                }
                else if (pTemplateRow["type"].ToString().Contains("phpbb"))
                {
                    pTemplateRow["myposturl"] = @"search\.php\?search_id=egosearch";
                    isFound = true;
                }
            }
            catch (WebException ex)
            {
                pTemplateRow["posturlstatus"] = "Error! " + _htmlWeb.StatusCode + ex.Message;
                return;
            }

            //Get message
            if (isFound)
                pTemplateRow["posturlstatus"] = "Found!";
            else
                pTemplateRow["posturlstatus"] = "Not Found! " + _htmlWeb.StatusCode;
        }

        /// <summary>
        /// Get Templates header in table ThreadKeywords
        /// </summary>
        /// <param name="title"></param>
        /// <param name="language"></param>
        /// <returns></returns>
        public string GetThreadName(string title, int language)
        {
            string sql = "select * from ThreadKeywords";
            DataTable db = MITI.BaseDBCe.ExecSql_DataSet(sql).Tables[0];
            //Use linq for datatable
            var result = (from keyword in db.AsEnumerable()
                          where keyword.Field<string>("Keyword").Trim().ToLower() == title.Trim().ToLower()
                          && keyword.Field<int>("Language") == language
                          select keyword["ThreadId"]).Distinct().ToList(); //distinct to get unique value

            //If not found then add
            foreach (string s in result)
            {
                return s;
            }
            return string.Empty;
        }

        /// <summary>
        /// Get user id in new thread url
        /// </summary>
        /// <param name="url"></param>
        public void GetThreadId()
        {
            pThreadTemplateRow["id"] = GetThreadIdByRegex(pThreadTemplateRow["url"].ToString());
        }

        /// <summary>
        /// Get user id in new thread url
        /// </summary>
        /// <param name="url"></param>
        public string GetThreadIdByRegex(string url)
        {
            string userid = "";
            Regex r = new Regex(@"(f=(?<id>\d+)|/(?<id>\d+)|showforum=(?<id>\d+))");
            Match m;

            m = r.Match(url);
            if (m.Success)
            {
                userid = m.Result("${id}");
            }
            else
            {
                List<string> list = new List<string>();
                list = url.Split(new char[]{'/'}).ToList();
                url = pTemplateRow["url"].ToString();
                foreach (string s in list)
                {
                    if (!url.Contains(s))
                        url += "/" + s;
                }
                
                HtmlAgilityPack.HtmlDocument doc = Get(url);
                if (doc != null)
                {
                    r = new Regex(@"newthread.php\?do=newthread\&amp\;f=(?<id>\d+)");
                    m = r.Match(doc.DocumentNode.InnerHtml);
                    if (m.Success)
                        userid = m.Result("${id}");
                }
            }
            return userid;
        }

        /// <summary>
        /// Get links in tables of a forum
        /// Samples xpath:
        /// .//table/tbody/tr/td/div[text() ='Threads']
        /// .//table[./tbody/tr/td/div[text() ='Threads']]/tbody/tr/td[2]
        /// .//table[count(./tbody/tr/td[5]) > 2]
        /// .//table[count(./tbody/tr/td[5]) > 10]/tbody/tr/td[2]
        /// .//table[count(./tbody/tr/td[5]) > 10]/tbody/tr/td[2]/div/a
        /// vBulletin & phpbb Samples
        /// .//table[count(./tbody/tr/td[5]) > 5]/tbody/tr/td[2]
        /// Search postion where is Forum, Last post, Threads, Posts
        /// .//table[count(./tbody/tr/td[5]) > 5]/tbody/tr[1]
        /// IPB Samples:
        /// .//table[count(./tbody/tr/td[3]) > 5]/tbody/tr/td[1]
        /// </summary>
        public Dictionary<string, string> GetLinksInTables(HtmlAgilityPack.HtmlDocument doc)
        {
            string url = "", title = "", xpath = "";
            HtmlNode href;
            bool isBreak = false;
            int index = 0;
            int nums = 0; //numbers of column in a row of a table
            Dictionary<string, string> links = new Dictionary<string,string>();
            string content = "";
            if (doc != null)
            {
                //Save html
                pTemplateRow["html"] = doc.DocumentNode.InnerHtml;
                //xpath = @".//table/tbody[count(./tr)>=3]/tr[count(./td)>=3]";
                //xpath = @".//table/tbody[count(./tr)>=3]/tr[count(./td)>=3]";
                //.//div[.//ul[.//a]]
                //.//div[./h2]
                xpath = @".//table/tbody[count(./tr)>=3]/tr[3][count(./td)>=3]";
                if (doc.DocumentNode.SelectNodes(xpath) == null)
                    xpath = @".//table[count(./tr)>=3]/tr[3][count(./td) >= 3]";

                //Indentify first column contains title of a table which the columns of a table >=3 
                if (doc.DocumentNode.SelectNodes(xpath) != null)
                {
                    foreach (HtmlNode tr in doc.DocumentNode.SelectNodes(xpath))
                    {
                        nums = tr.SelectNodes("./td").Count();
                        foreach (HtmlNode td in tr.SelectNodes("./td"))
                        {
                            //Get text() in href has string length > 10. Pattern example: <a href=http://..>This is sample text</a>
                            href = td.SelectSingleNode(".//a[string-length(.//text()) > '2']");
                            index++;
                            if (href != null)
                            {
                                isBreak = true;
                                break;
                            }
                        }
                        if (isBreak)
                            break;
                    }

                    HtmlNodeCollection nodeCollection;
                    //Get XPath of columns contains threads
                    xpath = @".//table/tbody[count(./tr/td[" + nums + "]) > 1]/tr/td[" + index + "]";
                    if (doc.DocumentNode.SelectNodes(xpath) == null)
                        xpath = @".//table[count(./tr/td[" + nums + "]) > 1]/tr/td[" + index + "]";
                    foreach (HtmlNode tr in doc.DocumentNode.SelectNodes(xpath))
                    {
                        nodeCollection = tr.SelectNodes(".//a[@href]");
                        if (nodeCollection != null)
                        {
                            foreach (HtmlNode td in nodeCollection)
                            {
                                url = td.Attributes["href"].Value.Replace("&amp;", "&");
                                title = td.ChildNodes[0].InnerHtml.Replace("&amp;", "&");
                                if (!links.ContainsKey(url))
                                    links.Add(url, title);
                            }
                        }

                        //get description in thread to indentify language of forum
                        foreach (HtmlNode node in tr.ChildNodes)
                        {
                            if ((node.Name == "div" || node.Name == "span" || node.Name == "p")
                                && node.ChildNodes.Count == 1)
                                content += node.InnerHtml + " ";
                        }
                    }
                }
                else
                {
                    if (doc.DocumentNode.SelectNodes("//a[@href]").Count > 0)
                    {
                        foreach (HtmlNode link in doc.DocumentNode.SelectNodes("//a[@href]"))
                        {
                            foreach (HtmlNode node in link.ChildNodes)
                            {
                                if (node.Name == "strong" || node.Name == "#text" || node.Name == "span")
                                {
                                    url = link.Attributes["href"].Value.Replace("&amp;", "&");
                                    title = node.InnerHtml.Replace("&amp;", "&");
                                    //Kiểm tra url thỏa mãn điều kiện
                                    if (!GeneralClass.CheckUrlKeyWord((int)enumKeyType.BlacklistUrl, url)
                                    && !GeneralClass.CheckTitleKeyWord((int)enumKeyType.BlacklistTitle, title))
                                    {
                                        //Kiểm tra xem nội dung bên trong node có empty không
                                        if (url != string.Empty && title != string.Empty)
                                        {
                                            if (!links.ContainsKey(url))
                                                links.Add(url, title);
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }
            links.Add("<text>", content);
            return links;
        }


        /// <summary>
        /// Get text to indentify language
        /// </summary>
        //public string GetTextInTables(HtmlAgilityPack.HtmlDocument doc)
        //{
        //    string url = "", title = "", xpath = "";
        //    HtmlNode href;
        //    bool isBreak = false;
        //    int index = 0;
        //    int nums = 0; //numbers of column in a row of a table
        //    //Dictionary<string, string> links = new Dictionary<string, string>();
        //    string content = "";
        //    if (doc != null)
        //    {
        //        //Save html
        //        pTemplateRow["html"] = doc.DocumentNode.InnerHtml;
        //        //xpath = @".//table/tbody[count(./tr)>=3]/tr[count(./td)>=3]";
        //        //xpath = @".//table/tbody[count(./tr)>=3]/tr[count(./td)>=3]";
        //        xpath = @".//table/tbody[count(./tr)>=3]/tr[3][count(./td)>=3]";
        //        if (doc.DocumentNode.SelectNodes(xpath) == null)
        //            xpath = @".//table[count(./tr)>=3]/tr[3][count(./td) >= 3]";

        //        //Indentify first column contains title of a table which the columns of a table >=3 
        //        if (doc.DocumentNode.SelectNodes(xpath) != null)
        //        {
        //            foreach (HtmlNode tr in doc.DocumentNode.SelectNodes(xpath))
        //            {
        //                nums = tr.SelectNodes("./td").Count();
        //                foreach (HtmlNode td in tr.SelectNodes("./td"))
        //                {
        //                    //Get text() in href has string length > 10. Pattern example: <a href=http://..>This is sample text</a>
        //                    href = td.SelectSingleNode(".//a[string-length(.//text()) > '2']");
        //                    index++;
        //                    if (href != null)
        //                    {
        //                        isBreak = true;
        //                        break;
        //                    }
        //                }
        //                if (isBreak)
        //                    break;
        //            }

        //            HtmlNodeCollection nodeCollection;
        //            //Get XPath of columns contains threads
        //            xpath = @".//table/tbody[count(./tr/td[" + nums + "]) > 1]/tr/td[" + index + "]";
        //            if (doc.DocumentNode.SelectNodes(xpath) == null)
        //                xpath = @".//table[count(./tr/td[" + nums + "]) > 1]/tr/td[" + index + "]";
        //            foreach (HtmlNode tr in doc.DocumentNode.SelectNodes(xpath))
        //            {
        //                foreach(HtmlNode node in tr.ChildNodes){
        //                    if (node.Name != "a")
        //                    {
        //                        content += node.InnerHtml + " ";
        //                        if (content.Length > 200)
        //                            return content;
        //                    }
        //                }
        //                //nodeCollection = tr.SelectNodes(".//a[@href]");
        //                //if (nodeCollection != null)
        //                //{
        //                //    foreach (HtmlNode td in nodeCollection)
        //                //    {
        //                //        url = td.Attributes["href"].Value.Replace("&amp;", "&"); ;
        //                //        title = td.ChildNodes[0].InnerHtml.Replace("&amp;", "&");
        //                //        if (!links.ContainsKey(url))
        //                //            links.Add(url, title);
        //                //    }
        //                //}
        //            }
        //        }
        //    }
        //    return links;
        //}

        ///// <summary>
        ///// Get id threads in a forum
        ///// </summary>
        //public void GetIdThreads(ref DataGridView dataGridViewCheckForums)
        //{
        //    //mpconfigDataSetTableAdapters.ThreadTemplatesTableAdapter threadTemplatesTableAdapter = new mpconfigDataSetTableAdapters.ThreadTemplatesTableAdapter();
        //    //threadTemplatesTableAdapter.Connection.ConnectionString = GeneralClass.ConnectString();
        //    //threadTemplatesTableAdapter.ClearBeforeFill = true;

        //    try
        //    {
        //        string url = "", title = "";
        //        int count = 1;
        //        DataGridViewRow dr;
        //        //Login
        //        if (!_isLogged)
        //            DoLogin();

        //        //Load Main page
        //        url = pTemplateRow["url"].ToString() + "/index.php";
        //        HtmlAgilityPack.HtmlDocument doc = Get(url);

        //        //Get links in tables of forum
        //        Dictionary<string, string> links = GetLinksInTables(doc);
        //        if (links.Count > 1)
        //        {
        //            foreach (KeyValuePair<string, string> val in links)
        //            {
        //                url = val.Key;
        //                title = val.Value;
        //                if (title.Length > 100)
        //                    title = title.Substring(0, 100);

        //                //dr = pThreadTemplateTable.NewRow();
        //                //dr["id"] = count++;
        //                //dr["forum_name"] = pTemplateRow["forum_name"];
        //                //dr["title"] = title;
        //                //dr["thread_name"] = GetThreadName(title, 12);
        //                //if (dr["thread_name"].ToString() != "")
        //                //    dr["thread_id"] = GetThreadIdByRegex(url);
        //                //dr["url"] = url;
        //                //dr["select"] = 0;
        //                //pThreadTemplateTable.Rows.Add(dr);
        //                //threadTemplatesBindingSource.EndEdit();
        //                //threadTemplatesTableAdapter.Update(dr);

        //                dr = new DataGridViewRow();
        //                dr.Cells["idDataGridViewTextBoxColumnThread"].Value = count++;
        //                dr.Cells["forumnameDataGridViewTextBoxColumnThread"].Value = pTemplateRow["forum_name"];
        //                dr.Cells["titleDataGridViewTextBoxColumnThread"].Value = title;
        //                dr.Cells["threadnameDataGridViewTextBoxColumnThread"].Value = GetThreadName(title, 12);
        //                if (dr.Cells["threadnameDataGridViewTextBoxColumnThread"].Value.ToString() != "")
        //                    dr.Cells["threadidDataGridViewTextBoxColumnThread"].Value = GetThreadIdByRegex(url);
        //                dr.Cells["urlDataGridViewTextBoxColumnThread"].Value = url;
        //                dr.Cells["selectDataGridViewTextBoxColumnThread"].Value = 0;
        //                dataGridViewCheckForums.Rows.Add(dr);
        //            }
        //        }
        //    }
        //    catch(WebException ex)
        //    {
        //        throw ex;
        //    }
        //}

        /// <summary>
        /// Get id threads in a forum
        /// </summary>
        //public void GetIdThreads(ref DataGridView dataGridViewCheckForums)
        public void GetIdThreads(mpconfigDataSet.ThreadTemplatesDataTable threadTemplatesDataTable)
        {
            //mpconfigDataSetTableAdapters.ThreadTemplatesTableAdapter threadTemplatesTableAdapter = new mpconfigDataSetTableAdapters.ThreadTemplatesTableAdapter();
            //threadTemplatesTableAdapter.Connection.ConnectionString = GeneralClass.ConnectString();
            //threadTemplatesTableAdapter.ClearBeforeFill = true;

            try
            {
                string url = "", title = "";
                int count = 1;
                
                //Login
                if (!_isLogged)
                    DoLogin();

                //Load Main page
                url = pTemplateRow["url"].ToString() + "/index.php";
                HtmlAgilityPack.HtmlDocument doc = Get(url);

                //Get links in tables of forum
                Dictionary<string, string> links = GetLinksInTables(doc);
                if (links.Count > 1)
                {
                    foreach (KeyValuePair<string, string> val in links)
                    {
                        url = val.Key;
                        title = val.Value;
                        if (title.Length > 100)
                            title = title.Substring(0, 100);

                        DataRow dr = threadTemplatesDataTable.NewRow();// pThreadTemplateTable.NewRow();
                        dr["id"] = count++;
                        dr["forum_name"] = pTemplateRow["forum_name"];
                        dr["title"] = title;
                        dr["thread_name"] = GetThreadName(title, 12);
                        if (dr["thread_name"].ToString() != "")
                            dr["thread_id"] = GetThreadIdByRegex(url);
                        dr["url"] = url;
                        dr["select"] = 0;
                        threadTemplatesDataTable.Rows.Add(dr);

                        //int index = dataGridViewCheckForums.Rows.Add();
                        //DataGridViewRow dr = dataGridViewCheckForums.Rows[index];
                        //dr.Cells["idDataGridViewTextBoxColumnThread"].Value = count++;
                        //dr.Cells["forumnameDataGridViewTextBoxColumnThread"].Value = pTemplateRow["forum_name"];
                        //dr.Cells["titleDataGridViewTextBoxColumnThread"].Value = title;
                        //dr.Cells["threadnameDataGridViewTextBoxColumnThread"].Value = GetThreadName(title, 12);
                        //if (dr.Cells["threadnameDataGridViewTextBoxColumnThread"].Value.ToString() != "")
                        //    dr.Cells["threadidDataGridViewTextBoxColumnThread"].Value = GetThreadIdByRegex(url);
                        //dr.Cells["urlDataGridViewTextBoxColumnThread"].Value = url;
                        //dr.Cells["selectDataGridViewTextBoxColumnThread"].Value = 0;
                        //dataGridViewCheckForums.Rows.Add(dr);
                    }
                }
            }
            catch (Exception ex)
            {
                throw ex;
            }
        }

        /// <summary>
        /// Get forum type in pattern by Regex
        /// Example: vBulletin 3.8.6, IB.Board 3.1.2, phpbb2, phpbb3
        /// </summary>
        /// <param name="content"></param>
        /// <param name="pattern"></param>
        /// <returns></returns>
        private string GetForumTypeByRegex(string content, string pattern)
        {
            string result = "#";
            int max = 0;
            List<string> list = new List<string>();
            Regex r = new Regex(pattern);
            Match m = r.Match(content);
            while (m.Success)
            {
                if (!list.Contains(m.Value))
                    list.Add(m.Value);
                m = m.NextMatch();
            }

            if (list.Count > 0)
            {
                foreach (string s in list)
                    if (s.Length > max)
                    {
                        result = s;
                        max = s.Length;
                    }
            }
            return result;
        }

        /// <summary>
        /// Return result in two key "type" and "forumtype" of Dictionary
        /// </summary>
        /// <param name="doc"></param>
        /// <returns></returns>
        public Dictionary<string, string> GetForumType(HtmlAgilityPack.HtmlDocument doc)
        {
            Dictionary<string, string> list = new Dictionary<string, string>();
            string vBulletin, phpBB, ipBoard;
            vBulletin = @"v[bB]ulletin((\&\#8482\;(</a>|)|\&reg\;|)\s*[vV]ersion|)\s*([0-9]\.[0-9]\.[0-9]|)"; //@"v[bB]ulletin((\&reg\;|)\s*[vV]ersion|)\s*([0-9]\.[0-9]\.[0-9]|)"; 
            phpBB = @"php[bB][pB]([23]|)";
            ipBoard = @"[iI][pP](\.|)[bB]oard(</a>|)\s*([0-9]\.[0-9]\.[0-9]|)";
            string result;
            list.Add("type", "");
            list.Add("forumtype", "#");

            //search vBulletin
            result = GetForumTypeByRegex(doc.DocumentNode.InnerHtml, vBulletin);
            if (result != "#")
            {
                list.Add("vBulletin", result);
            }

            //search phpBB is 2 or 3
            result = GetForumTypeByRegex(doc.DocumentNode.InnerHtml, phpBB);
            if (result.Contains("3"))
            {
                list.Add("phpbb3", result);
            }
            else if (result != "#")
            {
                string pattern = @"(/ucp\.php\?mode\=(login|register|logout))";
                Regex r = new Regex(pattern);
                Match m = r.Match(doc.DocumentNode.InnerHtml);
                if (m.Success)
                {
                    list.Add("phpbb3", "phpBB3");
                }
                else
                    list.Add("phpbb2", "phpBB2");
            }

            //search ipBoard
            result = GetForumTypeByRegex(doc.DocumentNode.InnerHtml, ipBoard);
            if (result != "#")
            {
                list.Add("IPB", result);
            }

            //return list if contains more than one value
            if (list.Count > 2)
            {
                string type = "";
                result = "";
                foreach (KeyValuePair<string, string> k in list)
                {
                    if (!k.Key.Contains("forumtype")
                        && !k.Key.Contains("type")
                        )
                    {
                        if (result != string.Empty)
                            result += "-" + k.Value;
                        else
                        {
                            result = k.Value;
                            //get first value for type column
                            type = k.Key;
                        }
                    }
                }

                list["type"] = type;
                Regex r = new Regex(@"((\&\#8482\;(</a>|)(?#textCopyright)|\&reg\;(</a>|)(?#textTM)|)\s*[vV]ersion|</a>)");
                Match m = r.Match(result);
                while (m.Success)
                {
                    result = result.Replace(m.Value, "");
                    m = m.NextMatch();
                }

                r = new Regex(@"[\t\r\n\s]+");
                m = r.Match(result);
                while (m.Success)
                {
                    result = result.Replace(m.Value, " ");
                    m = m.NextMatch();
                }
                //get value for forumtype column
                list["forumtype"] = result;
            }

            return list;
        }

        /// <summary>
        /// Use google translate to indentify language of a forum
        /// </summary>
        /// <param name="content"></param>
        /// <returns></returns>
        public string UseGoogleTranslateToFindLanguage(string content)
        {
            string url = String.Format("http://www.google.com/translate_t?hl=en&ie=UTF8&text={0}&langpair={1}", content, "auto|en");

            string result = String.Empty;

            using (WebClient webClient = new WebClient())
            {
                webClient.Encoding = System.Text.Encoding.UTF8;
                result = webClient.DownloadString(url);
            }
            //Regex r = new Regex(@"\w+ to English translation");
            //string language = r.Match(result).Value.Replace(" to English translation", "");
            Regex r = new Regex(@"(?<language>\w+) to English translation");
            string language = r.Match(result).Result("${language}");
            return language;
        }

        /// <summary>
        /// Indentify language of forum
        /// </summary>
        /// <param name="titles">string include titles</param>
        /// <returns></returns>
        public Dictionary<string, string> GetLanguage(HtmlAgilityPack.HtmlDocument doc)
        {
            Dictionary<string, string> result = new Dictionary<string, string>();
            //Init result
            result.Add("id", "1");
            result.Add("language", "#");

            //Indentify first column contains title of a table which the columns of a table >=3 
            Dictionary<string, string> links = GetLinksInTables(doc);
            //Get text to indentify langage
            string text = links["<text>"];
            //links.Remove("<text>");
            //string titles = string.Join(" ", links.Values.ToArray());
            if (text != "")
            {
                if (text.Length > 200)
                    text = text.Substring(0, 200);

                //User Google Translation to get Language
                string language = UseGoogleTranslateToFindLanguage(text);

                //2. Check if the language has been already stored in the database
                DataTable db = MITI.BaseDBCe.ExecSql_DataSet("select * from Languages").Tables[0];
                var query = (from keyword in db.AsEnumerable()
                             where keyword.Field<string>("language").Trim().ToLower() == language.ToLower()
                             select keyword["id"]).Distinct().ToList(); //distinct to get unique value

                if (query.Count() > 0)
                {
                    result["id"] = query[0].ToString();
                    result["language"] = language;
                }
            }

            return result;
        }

        /// <summary>
        /// Return numbers of keyword appear in text
        /// </summary>
        /// <param name="type"></param>
        /// <param name="lang"></param>
        /// <param name="text"></param>
        /// <returns></returns>
        public int CountKeyWord(int type, int lang, string text)
        {
            //Kiểm tra url có empty không
            if (text == string.Empty)
                return 0;
            DataTable db = MITI.BaseDBCe.ExecSql_DataSet("select * from Keywords").Tables[0];
            //Use linq for datatable
            var result = from keyword in db.AsEnumerable()
                         where keyword.Field<int>("KeyType") == type
                         && keyword.Field<int>("Language") == lang
                         && text.Trim().ToLower().Contains(keyword.Field<string>("KeyName").ToLower())
                         select keyword;
            return result.Count();
        }

        /// <summary>
        /// Indentify content of a forum is App, App-Porn or Porn
        /// </summary>
        /// <param name="doc"></param>
        /// <param name="language"></param>
        /// <returns></returns>
        public string IdentifyForumContentByKeyWords(string content, string language)
        {
            string result = "#";
            int lang = -1;
            //2. Check if the language has been already stored in the database
            DataTable db = MITI.BaseDBCe.ExecSql_DataSet("select * from Languages").Tables[0];
            var query = (from keyword in db.AsEnumerable()
                         where keyword.Field<string>("language").Trim().ToLower() == language.ToLower()
                         select keyword["id"]).Distinct().ToList(); //distinct to get unique value
            if (query.Count() > 0)
                lang = (int)query[0];

            //Check the forum is a application forum
            if (CountKeyWord(9, lang, content) > 3)
            {
                result = "App";
                //Check the forum is also a porn forum
                if (CountKeyWord(10, lang, content) > 3)
                {
                    result = "App-Porn";
                }
            }
            else
            {
                //Check the forum is also a porn forum
                if (CountKeyWord(10, lang, content) > 3)
                {
                    result = "Porn";
                }
            }
            return result;
        }

        /// <summary>
        /// Get statistics numbers of a forum, include:
        /// Total Threads (Topics)
        /// Total Posts
        /// Total Numbers
        /// Current Members
        /// Current Guests
        /// </summary>
        /// <param name="doc"></param>
        public void GetStatisticsNumbers(HtmlAgilityPack.HtmlDocument doc)
        {
            string pattern = "";
            Regex r;
            Match m;
            //GET TOTAL THREADS, MEMBERS, GUESTS
            HtmlToText htmltotext = new HtmlToText();
            string textcontent = htmltotext.ConvertHtmlDocument(doc);

            if (pTemplateRow["type"].ToString() == "vBulletin")
            {
                //CURRENT USERS: Samples: //1077 (35 members and 1042 guests)
                if (pTemplateRow["language"].ToString() == "English")
                {
                    pattern = @"(?<current_members>[\d\,]+) (M|m)embers";
                    pattern += @"\s*and\s*";
                    pattern += @"(?<current_guests>[\d\,]+) (G|g)uests";
                }
                else if (pTemplateRow["language"].ToString() == "Vietnamese")
                {
                    pattern = @"(?<current_members>[\d\,\.]+) (T|t)hành viên";
                    pattern += @"\s*và\s*";
                    pattern += @"(?<current_guests>[\d\,\.]+) (K|k)hách";
                }

                r = new Regex(pattern);
                //Match m = r.Match(doc.DocumentNode.InnerHtml);
                m = r.Match(textcontent);
                if (m.Success)
                {
                    pTemplateRow["current_members"] = m.Result("${current_members}").Replace(",", "");
                    pTemplateRow["current_guests"] = m.Result("${current_guests}").Replace(",", "");
                }

                //THREADS, POSTS, MEMBERS: Samples: //Threads: 236,429, Posts: 237,240, Members: 129,421 
                if (pTemplateRow["language"].ToString() == "English")
                {
                    pattern = @"(?<name1>(Threads|Posts|Members))(:|)[\s\\t\\r\\n]*(?<value1>[\d\,]+)";
                    pattern += @"[\s\\t\\r\\n]*";
                    pattern += @"(?<name2>(Threads|Posts|Members))(:|)[\s\\t\\r\\n]*(?<value2>[\d\,]+)";
                    pattern += @"[\s\\t\\r\\n]*";
                    pattern += @"(?<name3>(Threads|Posts|Members))(:|)[\s\\t\\r\\n]*(?<value3>[\d\,]+)";

                    r = new Regex(pattern);
                    //m = r.Match(doc.DocumentNode.InnerHtml);
                    m = r.Match(textcontent);
                    if (m.Success)
                    {
                        pTemplateRow[m.Result("${name1}").ToLower()] = m.Result("${value1}").Replace(",", "");
                        pTemplateRow[m.Result("${name2}").ToLower()] = m.Result("${value2}").Replace(",", "");
                        pTemplateRow[m.Result("${name3}").ToLower()] = m.Result("${value3}").Replace(",", "");
                    }
                }
                else if (pTemplateRow["language"].ToString() == "Vietnamese")
                {
                    pattern = @"(Chủ đề|Đề tài)(:|)[\s\\t\\r\\n]*(?<value1>[\d\,\.]+)";
                    pattern += @"[\s\\t\\r\\n]*";
                    pattern += @"Bài gửi(:|)[\s\\t\\r\\n]*(?<value2>[\d\,\.]+)";
                    pattern += @"[\s\\t\\r\\n]*";
                    pattern += @"Thành viên(:|)[\s\\t\\r\\n]*(?<value3>[\d\,\.]+)";

                    r = new Regex(pattern);
                    //m = r.Match(doc.DocumentNode.InnerHtml);
                    m = r.Match(textcontent);
                    if (m.Success)
                    {
                        pTemplateRow["threads"] = m.Result("${value1}").Replace(",", "").Replace(".", "");
                        pTemplateRow["posts"] = m.Result("${value2}").Replace(",", "").Replace(".", "");
                        pTemplateRow["members"] = m.Result("${value3}").Replace(",", "").Replace(".", "");
                    }
                }
            }
            else if (pTemplateRow["type"].ToString() == "phpbb2")
            {
                if (pTemplateRow["language"].ToString() == "English")
                {
                    //CURRENT USERS: Samples: 259 registered, 6 hidden and 336 guests (71 Registered, 1 Hidden and 440 Guests)
                    pattern = @"(?<current_members>[\d\,]+) (R|r)egistered";
                    pattern += @"\s*\,\s*";
                    pattern += @"(?<current_hidden>[\d\,]+) (H|h)idden";
                    pattern += @"\s*and\s*(about|)\s*";
                    pattern += @"(?<current_guests>[\d\,]+) (G|g)uests";
                    r = new Regex(pattern);
                    m = r.Match(textcontent);
                    if (m.Success)
                    {
                        pTemplateRow["current_members"] = m.Result("${current_members}").Replace(",", "");
                        pTemplateRow["current_guests"] = m.Result("${current_guests}").Replace(",", "");
                    }
                    else
                    {
                        //there are 647 users online
                        pattern = @"(T|t)here are (?<current_guests>[\d\,]+) (U|u)sers online";
                        r = new Regex(pattern);
                        m = r.Match(textcontent);
                        if (m.Success)
                            pTemplateRow["current_guests"] = m.Result("${current_guests}").Replace(",", "");
                    }

                    //THREADS, POSTS: Samples: Our users have posted a total of 1,914,978 articles (in 1,859,632 topics|)
                    pattern = @"a total of (?<value1>[\d\,]+) articles";
                    pattern += @"\s*(in (?<value2>[\d\,]+) topics|)";
                    r = new Regex(pattern);
                    //m = r.Match(doc.DocumentNode.InnerHtml);
                    m = r.Match(textcontent);
                    if (m.Success)
                    {
                        pTemplateRow["posts"] = m.Result("${value1}").Replace(",", "");
                        pTemplateRow["threads"] = m.Result("${value2}").Replace(",", "");
                    }
                    //MEMBERS: Samples: We have 21,525 registered users
                    pattern = @"We have (?<value>[\d\,]+) registered users";
                    r = new Regex(pattern);
                    m = r.Match(textcontent);
                    if (m.Success)
                    {
                        pTemplateRow["members"] = m.Result("${value}").Replace(",", "");
                    }
                }
            }
            else if (pTemplateRow["type"].ToString() == "phpbb3")
            {
                if (pTemplateRow["language"].ToString() == "English")
                {
                    //CURRENT USERS: Samples: 259 registered, 6 hidden and 336 guests
                    pattern = @"(?<current_members>[\d\,]+) (R|r)egistered";
                    pattern += @"\s*\,\s*";
                    pattern += @"(?<current_hidden>[\d\,]+) (H|h)idden";
                    pattern += @"\s*and\s*";
                    pattern += @"(?<current_guests>[\d\,]+) (G|g)uest(s|)";
                    r = new Regex(pattern);
                    m = r.Match(textcontent);
                    if (m.Success)
                    {
                        pTemplateRow["current_members"] = m.Result("${current_members}").Replace(",", "");
                        pTemplateRow["current_guests"] = m.Result("${current_guests}").Replace(",", "");
                    }

                    //THREADS, POSTS, MEMBERS: Samples: Total posts 3454410 • Total topics 3028845 • Total members 71598 
                    pattern = @"Total posts(:|)[\s\\t\\r\\n]*(?<value1>[\d\,]+)";
                    pattern += @"[\s\\t\\r\\n\|\•]*";
                    pattern += @"Total topics(:|)[\s\\t\\r\\n]*(?<value2>[\d\,]+)";
                    pattern += @"[\s\\t\\r\\n\|\•]*";
                    pattern += @"Total members(:|)[\s\\t\\r\\n]*(?<value3>[\d\,]+)";
                    r = new Regex(pattern);
                    //m = r.Match(doc.DocumentNode.InnerHtml);
                    m = r.Match(textcontent);
                    if (m.Success)
                    {
                        pTemplateRow["posts"] = m.Result("${value1}").Replace(",", "");
                        pTemplateRow["threads"] = m.Result("${value2}").Replace(",", "");
                        pTemplateRow["members"] = m.Result("${value3}").Replace(",", "");
                    }
                }
            }
            else if (pTemplateRow["type"].ToString() == "IPB")
            {
                if (pTemplateRow["language"].ToString() == "English")
                {
                    //CURRENT USERS: Samples: 5 members, 10 guests, 0 anonymous users 
                    pattern = @"(?<current_members>[\d\,]+) members";
                    pattern += @"\s*\,\s*";
                    pattern += @"(?<current_guests>[\d\,]+) guests";
                    r = new Regex(pattern);
                    m = r.Match(textcontent);
                    if (m.Success)
                    {
                        pTemplateRow["current_members"] = m.Result("${current_members}").Replace(",", "");
                        pTemplateRow["current_guests"] = m.Result("${current_guests}").Replace(",", "");
                    }

                    //THREADS, POSTS, MEMBERS: Samples: //Threads: 236,429, Posts: 237,240, Members: 129,421 
                    pattern = @"Total\s(?<name1>(Posts|Members))(:|)[\s\\t\\r\\n]*(?<value1>[\d\,]+)";
                    pattern += @"[\s\\t\\r\\n]*";
                    pattern += @"Total\s(?<name2>(Posts|Members))(:|)[\s\\t\\r\\n]*(?<value2>[\d\,]+)";
                    r = new Regex(pattern);
                    //m = r.Match(doc.DocumentNode.InnerHtml);
                    m = r.Match(textcontent);
                    if (m.Success)
                    {
                        pTemplateRow[m.Result("${name1}").ToLower()] = m.Result("${value1}").Replace(",", "");
                        pTemplateRow[m.Result("${name2}").ToLower()] = m.Result("${value2}").Replace(",", "");
                    }
                }
            }

        }

        /// <summary>
        /// Get content of forum:
        /// 1. Indentify language of a forum
        /// 2. Indentify content of a forum: App, App-Porn, Porn, ...
        /// 3. Indentify type of a forum: vBulletin, phpbb2, phpbb3, IPB
        /// 4. Indentify exactly forum type of a forum include version: vBulletin x.x.x, phpbb2, phpbb3, IP.Board x.x.x
        /// 5. Indentify statistics numbers of a forum: Total threads, posts, members, current members, current guests
        /// </summary>
        public void GetForumInformation()
        {
            try
            {
                //string titles = "", text = "";
                string content = "Unknown";

                if (!_isLogged)
                    DoLogin();

                HtmlAgilityPack.HtmlDocument doc = Get(pTemplateRow["url"].ToString() + "/index.php");

                //If not log in
                if (!_isLogged)
                {
                    //GET TYPE AND FORUMTYPE:
                    Dictionary<string, string> result = GetForumType(doc);
                    pTemplateRow["forumtype"] = result["forumtype"];
                    pTemplateRow["type"] = result["type"];
                }

                //GET LANGUAGE
                if (pTemplateRow["language"].ToString() == "")
                {
                    Dictionary<string, string> langDict = GetLanguage(doc);
                    pTemplateRow["language"] = langDict["language"];
                }
               

                //GET CONTENT FORUM
                if (!pTemplateRow["content"].ToString().Contains("App")
                    && !pTemplateRow["content"].ToString().Contains("Porn"))
                {
                    if (doc.DocumentNode.SelectNodes("//img") == null
                        || doc.DocumentNode.SelectNodes("//a[@href]") == null)
                    {
                        if (pTemplateRow["type"].ToString() != string.Empty)
                            content = "Unknown";
                        else
                            content = "Dead";
                    }
                    else if (doc.DocumentNode.SelectNodes("//img").Count() < 10
                            && doc.DocumentNode.SelectNodes("//a[@href]").Count() < 20)
                    {
                        if (pTemplateRow["type"].ToString() != string.Empty)
                            content = "Unknown";
                        else
                            content = "Domain";
                    }
                    else
                    {
                        //2. GET CONTENT: App, App-Porn, Porn, Foreign
                        if (pTemplateRow["language"].ToString() == "English")
                            content = IdentifyForumContentByKeyWords(doc.DocumentNode.InnerHtml, pTemplateRow["language"].ToString());
                        else
                            content = "Foreign";
                    }
                    pTemplateRow["content"] = content;
                }

                pTemplateRow["html"] = doc.DocumentNode.InnerHtml;
                pTemplateRow["message"] = DateTime.Now.ToString() + _htmlWeb.StatusCode;

                //Get statistics numbers: total threads, post, members, current members, current guest
                GetStatisticsNumbers(doc);
              
            }
            catch (WebException ex)
            {
                pTemplateRow["content"] = "Error!";
                pTemplateRow["message"] = DateTime.Now.ToString() + _htmlWeb.StatusCode + ex.Message;
            }
        }

#region old code
        ///// <summary>
        ///// Get user id in new thread url
        ///// </summary>
        ///// <param name="url"></param>
        //public string GetUserId(string url)
        //{
        //    string userid = "";
        //    Regex r = new Regex(@"\d{2,}");
        //    Match m = r.Match(url);
        //    if (m.Success)
        //        userid = m.Value;
        //    return userid;
        //}

        /// <summary>
        /// Tokenize a sentence to words use to indentiy language
        /// </summary>
        /// <param name="sentence"></param>
        /// <returns></returns>
        public void TokenizeSentence(List<string> list, string sentence)
        {
            //ArrayList filter = new ArrayList();
            //Regex re = new Regex(@"([a-zA-Z]\w+)\W*", RegexOptions.Compiled);
            Regex re = new Regex(@"\b[a-zA-Z]+\b", RegexOptions.Compiled);
            sentence = sentence.ToLower();
            Match m = re.Match(sentence);
            while (m.Success)
            {
                if (!list.Contains(m.Value))
                    list.Add(m.Value);
                m = m.NextMatch();
            }
        }

        /// <summary>
        /// Get forum content
        /// </summary>
        /// <param name="doc"></param>
        /// <returns></returns>
        //public string GetForumContent(HtmlAgilityPack.HtmlDocument doc)
        //{
        //    string content = "Unknown";
        //    try
        //    {
        //        if (doc.DocumentNode.SelectNodes("//img") == null
        //                || doc.DocumentNode.SelectNodes("//a[@href]") == null)
        //        {
        //            if (pTemplateRow["type"].ToString() != string.Empty)
        //                content = "Unknown";
        //            else
        //                content = "Dead";
        //        }
        //        else if (doc.DocumentNode.SelectNodes("//img").Count() < 10
        //                && doc.DocumentNode.SelectNodes("//a[@href]").Count() < 20)
        //        {
        //            if (pTemplateRow["type"].ToString() != string.Empty)
        //                content = "Unknown";
        //            else
        //                content = "Domain";
        //        }
        //        else
        //        {
        //            //2. GET CONTENT: App, App-Porn, Porn, Foreign
        //            if (pTemplateRow["language"].ToString() == "English")
        //                content = IdentifyForumContentByKeyWords(doc, pTemplateRow["language"].ToString());
        //            else
        //                content = "Foreign";
        //        }
        //    }
        //    catch (WebException ex)
        //    {
        //        throw ex;
        //    }
        //    return content;
        //}

        ///// <summary>
        ///// Get direct Id thread
        ///// Example: 
        ///// http://cyberwarez.info/forum/forumdisplay.php?17-Applications
        ///// http://www.katzforums.com/forumdisplay.php?f=23
        ///// id threads are 17 and 23
        ///// </summary>
        ///// <param name="url"></param>
        ///// <param name="pattern">[=|/|?][0-9]+</param>
        ///// <returns></returns>
        //public string GetDirectIdThread(string url, string pattern)
        //{
        //    string id = "";
        //    //Find string
        //    Regex rx = new Regex(pattern);
        //    Match m = rx.Match(url);
        //    //Get number
        //    Regex rx1 = new Regex("[0-9]+");
        //    Match m1 = rx1.Match(m.Value);

        //    id = m1.Value;
        //    return id;
        //}

        ///// <summary>
        ///// Check whether Url contains key word
        ///// </summary>
        ///// <param name="value"></param>
        ///// <returns></returns>
        //public bool CheckUrlKeyWord(int type, string key)
        //{
        //    //Kiểm tra url có empty không
        //    if (key == string.Empty)
        //        return true;
        //    //Use linq for datatable
        //    var result = from keyword in _Keywords.AsEnumerable()
        //                 where keyword.Field<int>("KeyType") == type
        //                 && key.ToLower().Contains(keyword.Field<string>("KeyName"))
        //                 select keyword;
        //    if (result.Count() > 0)
        //        return true;
        //    return false;
        //}
#endregion

        #endregion
    }

    /// <summary>
    /// Class FormElementCollection
    /// </summary>
    public class FormElementCollection : Dictionary<string, string>
    {
        #region original code

        

        public FormElementCollection(HtmlAgilityPack.HtmlDocument htmlDoc)
        {
            if (htmlDoc != null)
            {
                var inputs = htmlDoc.DocumentNode.Descendants("input");
                foreach (var element in inputs)
                {
                    AddInputElement(element);
                }

                var menus = htmlDoc.DocumentNode.Descendants("select");
                foreach (var element in menus)
                {
                    AddMenuElement(element);
                }

                var textareas = htmlDoc.DocumentNode.Descendants("textarea");
                foreach (var element in textareas)
                {
                    AddTextareaElement(element);
                }
            }
        }

       

        /// <summary>
        /// Assembles all form elements and values to POST. Also html encodes the values.  
        /// </summary>
        public string AssemblePostPayload()
        {
            string value;
            StringBuilder sb = new StringBuilder();
            foreach (var element in this)
            {
                value = System.Web.HttpUtility.UrlEncode(element.Value);
                sb.Append("&" + element.Key + "=" + value);
            }
            //Substring(1) to remove "&" 
            return sb.ToString().Substring(1);
        }


        private void AddInputElement(HtmlNode element)
        {
            string name = element.GetAttributeValue("name", "");
            string value = element.GetAttributeValue("value", "");
            string type = element.GetAttributeValue("type", "");

            if (string.IsNullOrEmpty(name)) return;

            switch (type.ToLower())
            {
                case "checkbox":
                case "radio":
                    if (!ContainsKey(name)) Add(name, "");
                    string isChecked = element.GetAttributeValue("checked", "unchecked");
                    if (!isChecked.Equals("unchecked")) this[name] = value;
                    break;
                default:
                    if (!ContainsKey(name))
                        Add(name, value);
                    break;
            }
        }

        private void AddMenuElement(HtmlNode element)
        {
            string name = element.GetAttributeValue("name", "");
            var options = element.Descendants("option");

            if (string.IsNullOrEmpty(name)) return;

            // choose the first option as default
            var firstOp = options.First();
            string defaultValue = firstOp.GetAttributeValue("value", firstOp.NextSibling.InnerText);

            Add(name, defaultValue);

            // check if any option is selected
            foreach (var option in options)
            {
                string selected = option.GetAttributeValue("selected", "notSelected");
                if (!selected.Equals("notSelected"))
                {
                    string selectedValue = option.GetAttributeValue("value", option.NextSibling.InnerText);
                    this[name] = selectedValue;
                }
            }
        }

        private void AddTextareaElement(HtmlNode element)
        {
            string name = element.GetAttributeValue("name", "");
            if (string.IsNullOrEmpty(name)) return;
            Add(name, element.InnerText);
        }

        #endregion

        #region my code
        public FormElementCollection()
        {

        }
        public FormElementCollection(ExtendedWebBrowser wb)
        {
            if (wb.Document != null)
            {
                //var inputs = htmlDoc.DocumentNode.Descendants("input");
                //foreach (var element in inputs)
                //{
                //    AddInputElement(element);
                //}

                //var menus = htmlDoc.DocumentNode.Descendants("select");
                //foreach (var element in menus)
                //{
                //    AddMenuElement(element);
                //}

                //var textareas = htmlDoc.DocumentNode.Descendants("textarea");
                //foreach (var element in textareas)
                //{
                //    AddTextareaElement(element);
                //}
            }

            if (wb.Document != null)
            {
                HtmlElementCollection inputs = wb.Document.GetElementsByTagName("input");
                foreach (HtmlElement element in inputs)
                {
                    AddInputElement(element);
                }

                HtmlElementCollection menus = wb.Document.GetElementsByTagName("select");
                foreach (HtmlElement element in menus)
                {
                    AddInputElement(element);
                }

                HtmlElementCollection textareas = wb.Document.GetElementsByTagName("textarea");
                foreach (HtmlElement element in textareas)
                {
                    AddInputElement(element);
                }
            }
        }

        private void AddInputElement(HtmlElement element)
        {
            string name = element.GetAttribute("name");
            string value = element.GetAttribute("value");
            string type = element.GetAttribute("type");

            if (string.IsNullOrEmpty(name)) return;

            switch (type.ToLower())
            {
                case "checkbox":
                case "radio":
                    if (!ContainsKey(name)) Add(name, "");
                    string isChecked = element.GetAttribute("checked");
                    if (!isChecked.Equals("unchecked")) this[name] = value;
                    break;
                default:
                    if (!ContainsKey(name))
                        Add(name, value);
                    break;
            }
        }

        private void AddMenuElement(HtmlElement element)
        {
            //string name = element.GetAttribute("name");
            //var options = element.GetElementsByTagName("option");

            //if (string.IsNullOrEmpty(name)) return;

            //// choose the first option as default
            //var firstOp = options..First();
            //string defaultValue = firstOp.GetAttributeValue("value", firstOp.NextSibling.InnerText);

            //Add(name, defaultValue);

            //// check if any option is selected
            //foreach (var option in options)
            //{
            //    string selected = option.GetAttributeValue("selected", "notSelected");
            //    if (!selected.Equals("notSelected"))
            //    {
            //        string selectedValue = option.GetAttributeValue("value", option.NextSibling.InnerText);
            //        this[name] = selectedValue;
            //    }
            //}
        }

        private void AddTextareaElement(HtmlElement element)
        {
            string name = element.GetAttribute("name");
            if (string.IsNullOrEmpty(name)) return;
            Add(name, element.InnerText);
        }

        private void SetInputElement(HtmlElement element)
        {
            try
            {
                string name = element.GetAttribute("name");
                string value = this[name];
                string type = element.GetAttribute("type");

                if (string.IsNullOrEmpty(name)) return;

                switch (type.ToLower())
                {
                    case "checkbox":
                    case "radio":
                        //if (!ContainsKey(name)) Add(name, "");
                        //string isChecked = element.GetAttribute("checked");
                        //if (!isChecked.Equals("unchecked")) this[name] = value;
                        break;
                    default:
                        element.SetAttribute("value", value);
                        break;
                }
            }
            catch
            {

            }
        }

        public void AutoFill(ExtendedWebBrowser wb)
        {
            HtmlElementCollection inputs = wb.Document.GetElementsByTagName("input");
            foreach (HtmlElement element in inputs)
            {
                SetInputElement(element);
            }

            HtmlElementCollection menus = wb.Document.GetElementsByTagName("select");
            foreach (HtmlElement element in menus)
            {
                SetInputElement(element);
            }

            HtmlElementCollection textareas = wb.Document.GetElementsByTagName("textarea");
            foreach (HtmlElement element in textareas)
            {
                SetInputElement(element);
            }
        }
       #endregion
    }
}
